Generating & Evaluating QA Pairs for Cleantech RAG Systems

%%writefile .env

api_key = "<your-api-key>"

!pip uninstall -y numpy
!pip install numpy==1.26.4

!pip install -q gensim==4.3.3 tiktoken seaborn spacy langchain openai sentence-transformers python-dotenv gdown pandas matplotlib nltk kagglehub

!python -m spacy download en_core_web_sm

import gensim
import gdown
import os
import warnings
from dotenv import load_dotenv
import pandas as pd
import numpy as np
import ast
from collections import Counter
import matplotlib.pyplot as plt
import seaborn as sns
import tiktoken
import spacy
from openai import OpenAI
from tqdm.auto import tqdm
from typing import Optional, List, Tuple
import json
import textwrap
from pathlib import Path

from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.docstore.document import Document as LangchainDocument

pd.set_option('display.max_colwidth', 100)
nlp = spacy.load("en_core_web_sm")

encoding = tiktoken.encoding_for_model("gpt-4")

import nltk
import string
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer, PorterStemmer

nltk.download('punkt_tab')
nltk.download('stopwords')
nltk.download('wordnet')

stop_words = set(stopwords.words('english'))
lemmatizer = WordNetLemmatizer()
stemmer = PorterStemmer()

warnings.filterwarnings("ignore")

load_dotenv()

client = OpenAI(
  base_url="https://openrouter.ai/api/v1",
  api_key=os.getenv("api_key"),
)

import kagglehub
from kagglehub import KaggleDatasetAdapter

file_path_media_dataset_v3_2024_10_28 = "cleantech_media_dataset_v3_2024-10-28.csv"

articles_df  = kagglehub.load_dataset(
  KaggleDatasetAdapter.PANDAS,
  "jannalipenkova/cleantech-media-dataset",
  file_path_media_dataset_v3_2024_10_28,
)

file_path_media_dataset_v3_2024_09_20 = "cleantech_rag_evaluation_data_2024-09-20.csv"

human_eval = kagglehub.load_dataset(
    KaggleDatasetAdapter.PANDAS,
    "jannalipenkova/cleantech-media-dataset",
    file_path_media_dataset_v3_2024_09_20,
    pandas_kwargs={
        "sep": ";",
        "engine": "python",
        "encoding": "latin-1",
    }
)

file_paths = [
    'bq-results-20240124-055833-1706076079048.json',
    'CleanTech_22-24.json',
    'CleanTech_22-24_updated.json'
]

dfs = {}

for file_path in file_paths:
   df = kagglehub.load_dataset(
       KaggleDatasetAdapter.PANDAS,
        "prakharbhandari20/cleantech-google-patent-dataset",
        "bq-results-20240124-055833-1706076079048.json",
        pandas_kwargs={
            "lines": True,
        })
   dfs[file_path] = df



patent_df = dfs['CleanTech_22-24_updated.json']

file_id = "1hT_VCmRUEw_AN4AzRBJHEKSBHnUpD6Oo"
url = f"https://drive.google.com/uc?id={file_id}"
gdown.download(url, output="datasets.zip", quiet=False)

!unzip -q datasets.zip

import zipfile
with zipfile.ZipFile("datasets.zip", "r") as zip_file:
    zip_file.extractall()
!rm datasets.zip

articles_df.head(5)

patent_df.head(3)

def preprocess_text(text, apply_stemming=False):
    """
    Preprocess a given text by:
      - Converting to lowercase
      - Tokenizing
      - Removing punctuation and stop words
      - Applying lemmatization (and optionally stemming)
    """
    if not isinstance(text, str):
        return []

    text = text.lower()
    tokens = word_tokenize(text)
    tokens = [token for token in tokens if token not in string.punctuation]
    tokens = [token for token in tokens if token not in stop_words]
    tokens = [lemmatizer.lemmatize(token) for token in tokens]

    if apply_stemming:
        tokens = [stemmer.stem(token) for token in tokens]

    return tokens

articles_df['tokens_title'] = articles_df['title'].apply(preprocess_text)
articles_df['tokens_content'] = articles_df['content'].apply(preprocess_text)

patent_df['tokens_title'] = patent_df['title_localized'].apply(
    lambda lst: preprocess_text(lst[0]['text']) if isinstance(lst, list) and lst and 'text' in lst[0] else []
)

patent_df['tokens_abstract'] = patent_df['abstract_localized'].apply(
    lambda lst: preprocess_text(lst[0]['text']) if isinstance(lst, list) and lst and 'text' in lst[0] else []
)

from collections import Counter

media_tokens  = [tok for toks in articles_df['tokens_content'] for tok in toks]
patent_tokens = [tok for toks in patent_df['tokens_abstract']  for tok in toks]

media_top  = Counter(media_tokens).most_common(15)
patent_top = Counter(patent_tokens).most_common(15)

m_terms, m_counts = zip(*media_top)
p_terms, p_counts = zip(*patent_top)

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 6))

ax1.barh(m_terms[::-1], m_counts[::-1])
ax1.set_title("Top 15 Tokens in Media Content")
ax1.invert_yaxis()

ax2.barh(p_terms[::-1], p_counts[::-1])
ax2.set_title("Top 15 Tokens in Patent Abstracts")
ax2.invert_yaxis()

plt.tight_layout()
plt.show()

def show_questions(title: str, questions: list, n: int = 6) -> None:
    """
    Display the first n questions from the given list with a title.

    Parameters:
    - title: A string to display as the category title.
    - questions: A list of questions.
    - n: The maximum number of questions to display (default is 6).
    """
    print(title)
    print("-" * len(title))
    for idx, question in enumerate(questions[:n], start=1):
        print(f"{idx}. {question}")
    print()



def gpt4_tokenizer(text: str):
    """
    Tokenizes the text using the GPT-4 tokenizer from tiktoken.
    Returns a list of token strings.
    """
    token_ids = encoding.encode(text)
    tokens = [encoding.decode([tid]) for tid in token_ids]
    return tokens

def plot_character_distribution(df):
    """
    Plots a histogram (with KDE) of the character lengths in the 'question' column.
    Displays vertical lines for the median and mean character counts.
    """
    char_lengths = df["question"].map(len)
    sns.histplot(char_lengths, kde=True)
    plt.title("Amount of characters in questions")
    plt.xlabel("Amount of characters")
    plt.ylabel("Number of questions")

    median_char_len = char_lengths.median()
    mean_char_len = char_lengths.mean()
    plt.axvline(median_char_len, color='r', linestyle='--', label=f"Median: {median_char_len:.2f}")
    plt.axvline(mean_char_len, color='g', linestyle='--', label=f"Mean: {mean_char_len:.2f}")
    plt.legend()
    plt.show()

def plot_word_distribution(df):
    """
    Plots a histogram (with KDE) of the word counts in the 'question' column.
    Displays vertical lines for the median and mean word counts.
    """
    word_counts = df["question"].map(lambda x: len(x.split()))
    sns.histplot(word_counts, kde=True)
    plt.title("Amount of words in questions")
    plt.xlabel("Amount of words")
    plt.ylabel("Number of questions")

    median_word_count = word_counts.median()
    mean_word_count = word_counts.mean()
    plt.axvline(median_word_count, color='r', linestyle='--', label=f"Median: {median_word_count:.2f}")
    plt.axvline(mean_word_count, color='g', linestyle='--', label=f"Mean: {mean_word_count:.2f}")
    plt.legend()
    plt.show()

def plot_gpt4_token_distribution(df):
    """
    Uses the GPT-4 tokenizer to tokenize the 'question' column,
    filters out non-alphabetic tokens, counts the top 20 most common tokens,
    and displays a horizontal barplot.
    """
    questions = df['question'].tolist()

    all_tokens = [token for question in questions for token in gpt4_tokenizer(question)]

    alpha_tokens = [token.strip().lower() for token in all_tokens if token.strip().isalpha()]

    alpha_token_counts = Counter(alpha_tokens)
    top_tokens = alpha_token_counts.most_common(20)

    if top_tokens:
        tokens, counts = zip(*top_tokens)
        sns.barplot(x=list(counts), y=list(tokens))
        plt.title("Most common alphabetic tokens")
        plt.xlabel("Count")
        plt.ylabel("Token")
        plt.show()
    else:
        print("No tokens found after filtering.")

def plot_non_stopword_token_distribution(df):
    """
    Uses the GPT-4 tokenizer to tokenize the 'question' column,
    filters out non-alphabetic tokens and then filters out stopwords using spaCy.
    It then counts the top 20 most common non-stopword tokens and displays a horizontal barplot.
    """
    questions = df['question'].tolist()

    all_tokens = [token for question in questions for token in gpt4_tokenizer(question)]
    alpha_tokens = [token.strip().lower() for token in all_tokens if token.strip().isalpha()]
    non_stop_tokens = [token for token in alpha_tokens if not nlp.vocab[token].is_stop]
    non_stop_token_counts = Counter(non_stop_tokens)
    top_tokens = non_stop_token_counts.most_common(20)

    if top_tokens:
        tokens, counts = zip(*top_tokens)
        sns.barplot(x=list(counts), y=list(tokens))
        plt.title("Most common non-stopword tokens")
        plt.xlabel("Count")
        plt.ylabel("Token")
        plt.show()
    else:
        print("No non-stop tokens found after filtering.")

def printwrap(text, wrap_lenght = 80):
  wrapped_text = textwrap.fill(text,wrap_lenght)
  return wrapped_text

azocleantech_df = articles_df[articles_df['domain']=='azocleantech']

solarindustrymag_df = articles_df[articles_df['domain']=='solarindustrymag']

pv_magazine_df = articles_df[articles_df['domain']=='pv-magazine']

energyvoice_df = articles_df[articles_df['domain']=='energyvoice']

cleantechnica_df = articles_df[articles_df['domain']=='cleantechnica']

# 7 paragraphs of azocleantech_df
selected_indices = [17690, 17915, 18349, 17715, 19247, 15788, 2195]
paragraphs_azocleantech_df = azocleantech_df.loc[selected_indices]

paragraphs_azocleantech_df.head(2)

# 8 paragraphs of solarindustrymag_df

selected_indices = [14,4307,1513,2216,13545,13382,2344,1578]
paragraphs_solarindustrymag_df = solarindustrymag_df.loc[selected_indices]

paragraphs_solarindustrymag_df.head(2)

pv_magazine_indices=[1311, 1416, 14590, 16840, 17599, 17643, 18321, 18854, 18906, 19194]
paragraphs_pv_magazine_df = pv_magazine_df.loc[pv_magazine_indices]

paragraphs_pv_magazine_df.head(2)

energyvoice_indices=[870, 2204, 2979, 3255, 4605, 5268, 6320, 12599, 12647, 18538]
paragraphs_energyvoice_df = energyvoice_df.loc[energyvoice_indices]

paragraphs_energyvoice_df.head(2)

filtered_indices = cleantechnica_df.index[cleantechnica_df["Unnamed: 0"].isin(
    [95438, 95392, 95363, 95261, 95173, 95163, 95133, 95106,
     95074, 94927, 94925, 94662, 94668, 94645, 95411]
)].tolist()

paragraphs_cleantechnica_df = cleantechnica_df.loc[filtered_indices]

paragraphs_cleantechnica_df.head(2)

combined_df = pd.concat([paragraphs_azocleantech_df,
                         paragraphs_solarindustrymag_df,
                         paragraphs_pv_magazine_df,
                         paragraphs_energyvoice_df,
                         paragraphs_cleantechnica_df],
                        ignore_index=True)

data_folder = Path("./data")
if not data_folder.exists():
    data_folder.mkdir()

combined_df.to_csv("data/paragraphs_media_dataset.csv")

combined_df.head(3)

from openai import OpenAI
import pandas as pd
import re

def generate_qa_pairs_media(df, target_qa_pairs=43, dataset_type="media",model = "meta-llama/llama-3.2-3b-instruct"):
    """
    Generates QA pairs from a given DataFrame using Llama 3.2 Instruct.

    Parameters:
      df (pd.DataFrame): DataFrame containing the reference texts.
         For the media dataset, it is expected to have a 'content' column and 'url'.
         For the patent dataset, it is expected to have an 'abstract' column, 'title',
         'publication_date', and 'inventor'.
      target_qa_pairs (int): The target number of QA pairs to generate.
      dataset_type (str): Specifies the dataset type. Use "media" for the media dataset,
         and "patent" for the patent dataset.

    Returns:
      pd.DataFrame: DataFrame containing the generated QA pairs and the associated metadata.
    """
    qa_pairs = []

    for _, row in df.iterrows():
        if dataset_type.lower() == "media":
            context_text = row['content']
        elif dataset_type.lower() == "patent":
            context_text = row['abstract']
        else:
            raise ValueError("dataset_type must be either 'media' or 'patent'.")

        paragraphs = context_text.split("\n\n")

        for paragraph in paragraphs:
            if not paragraph.strip():
                continue

            prompt = (
            "Using the following text, generate 15 unique and meaningful question–answer pairs.\n\n"
            "Instructions:\n"
            "1. Questions should be varied: include what, who, when, where, why, and how as appropriate.\n"
            "2. Focus on important details, facts, or concepts mentioned in the text.\n"
            "3. Provide clear answers that explain the key idea without being overly brief or too long.\n"
            "4. Format each pair as follows:\n"
            "Q: [Your question here]\n"
            "A: [Your answer here]\n\n"
            "Separate each Q&A pair with a blank line.\n\n"
            f"Text:\n{paragraph.strip()}"
        )


            try:
                completion = client.chat.completions.create(
                    model=model,
                    messages=[{"role": "user", "content": prompt}],
                )

                response = completion.choices[0].message.content.strip()

                # Extract Q&A pairs from the response
                qa_text = re.findall(r"Q:\s*(.*?)\s*A:\s*(.*?)(?=\nQ:|\Z)", response, re.DOTALL)

                for question, answer in qa_text:
                    if not any(existing["question"] == question for existing in qa_pairs):
                        if dataset_type.lower() == "media":
                            qa_pair = {
                                "question": question.strip(),
                                "relevant_text": paragraph.strip(),
                                "answer": answer.strip(),
                                "article_url": row.get('url', '')
                            }
                        else:
                            qa_pair = {
                                "question": question.strip(),
                                "relevant_text": paragraph.strip(),
                                "answer": answer.strip(),
                                "title": row.get('title', ''),
                                "publication_date": row.get('publication_date', ''),
                                "inventor": row.get('inventor', '')
                            }

                        qa_pairs.append(qa_pair)

                    if len(qa_pairs) >= target_qa_pairs:
                        break

            except Exception as e:
                print(f"Error: {e}")
                continue

        if len(qa_pairs) >= target_qa_pairs:
            break

    return pd.DataFrame(qa_pairs)

media_qa_df_llama = generate_qa_pairs_media(combined_df, target_qa_pairs=800, dataset_type="media")
media_qa_df_llama.to_csv("data/media_qa_df_llama.csv", index= False)

media_qa_df_llama = pd.read_csv("datasets/media_qa_df_llama.csv")

media_qa_df_llama.head(5)[["question","answer"]]

# patent_df_related = pd.read_csv('datasets/patent_df_selected_abstracts.csv')

patent_df.head(3)

patent_df['title_lang'] = patent_df['title_localized'].apply(lambda x:x[0]['language'])
patent_df['abstract_lang'] = patent_df['abstract_localized'].apply(lambda x:x[0]['language'])

patent_df['title_text'] = patent_df['title_localized'].apply(lambda x:x[0]['text'])
patent_df['abstract_text'] = patent_df['abstract_localized'].apply(lambda x:x[0]['text'])

patent_df['total'] = patent_df['title_text']+patent_df['abstract_text']

patent_df['w_count'] = patent_df['total'].apply(lambda x:len(x.split()))

patent_df.head(2)

plt.figure(figsize=(10, 6))
sns.histplot(patent_df['w_count'], kde=True)
plt.title('Distribution of Word Counts in Patents')
plt.xlabel('Word Count')
plt.ylabel('Frequency')
plt.show()

display(patent_df[['title_lang','abstract_lang']].value_counts())

lang_pairs = (
    patent_df[['title_lang', 'abstract_lang']]
    .value_counts()
    .reset_index(name='count')
    .assign(pair=lambda df: df['title_lang'] + '→' + df['abstract_lang'])
)

plt.figure(figsize=(8, 6))
sns.barplot(
    data=lang_pairs.nlargest(10, 'count'),
    x='count',
    y='pair'
)
plt.title("Top 10 Patent Title→Abstract Language Pairs")
plt.xlabel("Number of Patents")
plt.ylabel("Language Pair")
plt.tight_layout()
plt.show()

clean_patent_df = patent_df[(patent_df['title_lang'] == 'en')*(patent_df['abstract_lang'] == 'en')*(patent_df['w_count']>20)]

len(clean_patent_df)

plt.figure(figsize=(10, 6))
sns.kdeplot(patent_df['w_count'], label='Original', bw_adjust=1)
sns.kdeplot(clean_patent_df['w_count'], label='Filtered (w_count>20)', bw_adjust=1)
plt.title("Patent Word-Count Distribution: Original vs. Filtered")
plt.xlabel("Word Count")
plt.ylabel("Density")
plt.legend()
plt.show()

import gensim
from gensim.utils import simple_preprocess
from gensim.parsing.preprocessing import STOPWORDS
from gensim import corpora



clean_patent_df['clean_tokens'] = clean_patent_df['total'].apply(preprocess_text)
processed_docs = clean_patent_df['clean_tokens']

# create a dictionary and corpus
dictionary = corpora.Dictionary(processed_docs)
corpus = [dictionary.doc2bow(doc) for doc in processed_docs]

# apply LDA topic modeling
lda_model = gensim.models.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=10, passes=10, workers=4)

for idx, topic in lda_model.print_topics(-1):
    print(f"Topic: {idx} \nWords: {topic}\n")

# find the dominant topic for each document
for index, row in clean_patent_df.iterrows():
    bow = dictionary.doc2bow(row['clean_tokens'])
    dominant_topic = max(lda_model[bow], key=lambda item: item[1])

def get_top_words_and_docs(lda_model, corpus, dictionary, num_topics=10, top_words=10, top_docs=5):
    topic_results = []
    for topic_idx in range(num_topics):
        # get top words for the topic
        top_words_for_topic = [word for word, prob in lda_model.show_topic(topic_idx, top_words)]

        # get the documents with the highest probability for this topic
        topic_document_probs = []  # list to store document probabilities for this topic
        for doc_idx, doc_bow in enumerate(corpus):
          for topic_prob_idx, topic_prob in lda_model[doc_bow]:
            if topic_prob_idx == topic_idx:
              topic_document_probs.append((doc_idx, topic_prob))
              break  # move on to next document

        topic_document_probs.sort(key = lambda x: x[1], reverse=True)  # sort in descending order based on probability

        top_docs_for_topic = [doc_idx for doc_idx, _ in topic_document_probs[:top_docs]]


        topic_results.append({
            'topic_id': topic_idx,
            'top_words': top_words_for_topic,
            'top_documents': top_docs_for_topic
        })
    return topic_results


topic_results = get_top_words_and_docs(lda_model, corpus, dictionary)
document_indices = []
for topic in topic_results:
  print(f"Topic {topic['topic_id']}:")
  print("Top Words:", topic['top_words'])
  print("Top Documents:", topic['top_documents'])
  document_indices.extend(topic['top_documents'])
  print("-"*20)

patent_df_selected_abstracts = clean_patent_df.iloc[document_indices]

patent_df_selected_abstracts.head(2)

patent_df_selected_abstracts.to_csv('data/patent_df_selected_abstracts.csv',index=False)

patent_df_selected_abstracts = pd.read_csv("datasets/patent_df_selected_abstracts.csv")

import json
import re


def extract_json_from_text(text):
    # try to find the JSON block (between the first [ and last ])
    match = re.search(r"\[\s*{.*?}\s*]", text, re.DOTALL)
    if match:
        return match.group(0)
    else:
        return None

def generate_qa_pairs_patent(df, target_qa_pairs=43,model = "meta-llama/llama-3.2-3b-instruct"):
    qa_pairs = []
    for _ in range(target_qa_pairs):
      for _, row in tqdm(df.iterrows()):
          abstract = row['total']
          prompt = f"""
          You are a helpful assistant. Extract exactly 3 useful and meaningful question and answer pairs from the following text:


          {abstract}


          Return your answer in this JSON format:
        [
          {{
            "question": "Your first question?",
            "answer": "The answer to the first question."
          }},
          {{
            "question": "Your second question?",
            "answer": "The answer to the second question."
          }},
          {{
            "question": "Your third question?",
            "answer": "The answer to the third question."
          }}
        ]
          """
          # try:
          response = client.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": prompt}]
          )

          response_text = response.choices[0].message.content
          json_block = extract_json_from_text(response_text)
          if not json_block:
              print("❌ Could not find JSON in response:")
              print(response_text)
              continue

          try:
              qa_list = json.loads(json_block)
              for qa in qa_list:
                  qa_pairs.append({
                      "question": qa["question"],
                      "answer": qa["answer"],
                      "relevant_text": abstract,
                      "title": row["title_text"],
                      "publication_date": row["publication_date"],
                      "inventor": row["inventor"]
                  })
          except json.JSONDecodeError:
              print("❌ JSON parsing failed. Here's what the model returned:")
              print(json_block)
              continue
    return pd.DataFrame(qa_pairs)

patent_df_related = pd.read_csv('data/patent_df_selected_abstracts.csv')

patents_qa_df_llama = pd.read_csv("datasets/patents_qa_df_510_llama_new.csv")

patents_qa_df_llama = generate_qa_pairs_patent(patent_df_related, target_qa_pairs=4)

patents_qa_df_llama.head(5)[['question','answer']]

patents_qa_df_llama.to_csv('patents_qa_df_510_llama.csv', index=False)

models = client.models.list()
for model in models:
    if 'deepseek' in model.id:
        print(model.id)

MODEL = 'deepseek/deepseek-chat-v3-0324'

media_qa_df_deepseek = generate_qa_pairs_media(combined_df, target_qa_pairs=50, dataset_type="media",model = MODEL)
media_qa_df_deepseek.to_csv("data/media_qa_pairs_deepseek.csv", index= False)
media_qa_df_deepseek.head()[['question','answer']]

media_qa_df_deepseek = pd.read_csv("datasets/media_qa_pairs_deepseek.csv")

media_qa_df_deepseek.head(4)[['question','answer']]

patents_qa_df_deepseek = generate_qa_pairs_patent(patent_df_related, target_qa_pairs=1,model=MODEL)

patents_qa_df_deepseek.to_csv('data/patents_qa_df_deepseek.csv', index=False)

patents_qa_df_deepseek = pd.read_csv('datasets/patents_qa_df_deepseek.csv')

patents_qa_df_deepseek.head(4)[['question','answer']]

counts = {
    'Media (Llama)':       len(media_qa_df_llama),
    'Media (Deepseek)':    len(media_qa_df_deepseek),
    'Patents (Llama)':     len(patents_qa_df_llama),
    'Patents (Deepseek)':  len(patents_qa_df_deepseek),
}

plt.figure(figsize=(6,4))
plt.bar(counts.keys(), counts.values())
plt.title("Total QA Pairs by Domain & Method")
plt.ylabel("Number of QA Pairs")
plt.xticks(rotation=45, ha='right')
plt.tight_layout()
plt.show()

patent_qa_df_llama = pd.read_csv("datasets/patents_qa_df_510_llama_new.csv")
media_qa_df_llama = pd.read_csv("datasets/media_qa_df_llama.csv")

patents_qa_df = [patent_qa_df_llama]
media_qa_df = [media_qa_df_llama]

def classify_question_rule_based(question, categories):
    question_lower = question.lower()
    scores = {cat: 0 for cat in categories}
    for cat, keywords in categories.items():
        for kw in keywords:
            if kw in question_lower:
                scores[cat] += 1
    return max(scores, key=scores.get)

categories = {
    "Factual Questions": ["what", "when", "where", "who", "which", "fact"],
    "Comparative & Market Analysis Questions": ["compare", "contrast", "market", "trend", "analysis"],
    "Analytical & Explanatory Questions": ["why", "how", "explain", "impact"],
    "Government & Corporate Initiatives": ["government", "policy", "corporate", "initiative", "regulation"],
    "Sustainability & Technological Innovation Questions": ["sustainable", "innovation", "technology", "renewable", "eco-friendly"]
}

def stack_and_categorize(dfs, categories):
    df_combined = pd.concat(dfs, ignore_index=True)
    df_combined['category'] = df_combined['question'].apply(lambda q: classify_question_rule_based(q, categories))
    return df_combined

df_patent_categorized_rule_based = stack_and_categorize(patents_qa_df, categories)
df_media_categorized_rule_based = stack_and_categorize(media_qa_df, categories)

df_patent_categorized_rule_based.head(3)

df_media_categorized_rule_based.head(3)

from sentence_transformers import SentenceTransformer, util
import torch

prototypes = {
    "Factual Questions": "Questions that ask for specific facts or details.",
    "Comparative & Market Analysis Questions": "Questions that compare options or analyze market trends.",
    "Analytical & Explanatory Questions": "Questions that require detailed analysis or explanation.",
    "Government & Corporate Initiatives": "Questions related to government policies or corporate initiatives.",
    "Sustainability & Technological Innovation Questions": "Questions focusing on sustainability and innovative technology."
}

model = SentenceTransformer('all-MiniLM-L6-v2')

prototype_texts = list(prototypes.values())
prototype_labels = list(prototypes.keys())
prototype_embeddings = model.encode(prototype_texts, convert_to_tensor=True)

def classify_question_embedding(question: str) -> str:
    """
    Classify a question into one of the five categories using cosine similarity
    between its embedding and the precomputed prototype embeddings.
    """
    question_embedding = model.encode(question, convert_to_tensor=True)
    cosine_scores = util.cos_sim(question_embedding, prototype_embeddings)
    best_idx = torch.argmax(cosine_scores).item()
    return prototype_labels[best_idx]

def stack_and_categorize(dfs: list) -> pd.DataFrame:
    """
    Stack multiple DataFrames and add a new 'category' column by classifying each question.
    """
    df_combined = pd.concat(dfs, ignore_index=True)
    df_combined['category'] = df_combined['question'].apply(lambda q: classify_question_embedding(q))
    return df_combined

df_patent_categorized_sentence_bert = stack_and_categorize(patents_qa_df)
df_media_categorized_sentence_bert = stack_and_categorize(media_qa_df)

df_patent_categorized_sentence_bert.head(3)

df_media_categorized_sentence_bert.head(3)

model = SentenceTransformer('AI-Growth-Lab/PatentSBERTa')

prototype_texts = list(prototypes.values())
prototype_labels = list(prototypes.keys())
prototype_embeddings = model.encode(prototype_texts, convert_to_tensor=True)

df_patent_categorized_patent_sentence_bert = stack_and_categorize(patents_qa_df)
df_media_categorized_patent_sentence_bert= stack_and_categorize(media_qa_df)

df_patent_categorized_patent_sentence_bert.head(3)

df_media_categorized_patent_sentence_bert.head(3)

categories_rule_based = df_patent_categorized_rule_based['category'].value_counts()

barplot = sns.barplot(
    x=categories_rule_based.values,
    y=categories_rule_based.index,
    hue=categories_rule_based.index
)

barplot.set_title('Question Count by Categories')
barplot.set_xlabel('Question Count')
barplot.set_ylabel('Category')

plt.show()

categories_sentence_bert = df_media_categorized_sentence_bert['category'].value_counts()

barplot = sns.barplot(
    x=categories_sentence_bert.values,
    y=categories_sentence_bert.index,
    hue=categories_sentence_bert.index
)

barplot.set_title('Question Count by Categories')
barplot.set_xlabel('Question Count')
barplot.set_ylabel('Category')

plt.show()

categories_patent_bert = df_media_categorized_patent_sentence_bert['category'].value_counts()

barplot = sns.barplot(
    x=categories_patent_bert.values,
    y=categories_patent_bert.index,
    hue=categories_patent_bert.index
)

barplot.set_title('Question Count by Categories')
barplot.set_xlabel('Question Count')
barplot.set_ylabel('Category')

plt.show()

factual_questions = df_media_categorized_sentence_bert [df_media_categorized_sentence_bert ['category'] == 'Factual Questions']['question'].tolist()
factual_df = df_media_categorized_sentence_bert [df_media_categorized_sentence_bert ['category'] == 'Factual Questions']

show_questions("Factual Questions for the Media Dataset", factual_questions)

plot_character_distribution(factual_df)
plot_word_distribution(factual_df)
plot_gpt4_token_distribution(factual_df)
plot_non_stopword_token_distribution(factual_df)

factual_questions = df_patent_categorized_sentence_bert  [df_patent_categorized_sentence_bert  ['category'] == 'Factual Questions']['question'].tolist()
factual_df = df_patent_categorized_sentence_bert  [df_patent_categorized_sentence_bert  ['category'] == 'Factual Questions']
show_questions("Factual Questions", factual_questions)
plot_character_distribution(factual_df)
plot_word_distribution(factual_df)
plot_gpt4_token_distribution(factual_df)
plot_non_stopword_token_distribution(factual_df)

Comparative_Market_Analysis_Questions = df_media_categorized_sentence_bert [df_media_categorized_sentence_bert ['category'] == 'Comparative & Market Analysis Questions']['question'].tolist()
Comparative_Market_Analysis_df = df_media_categorized_sentence_bert [df_media_categorized_sentence_bert ['category'] == 'Comparative & Market Analysis Questions']

show_questions("Comparative & Market Analysis Questions for the Media Dataset", Comparative_Market_Analysis_Questions)

plot_character_distribution(Comparative_Market_Analysis_df)
plot_word_distribution(Comparative_Market_Analysis_df)
plot_gpt4_token_distribution(Comparative_Market_Analysis_df)
plot_non_stopword_token_distribution(Comparative_Market_Analysis_df)

Comparative_Market_Analysis_Questions = df_patent_categorized_sentence_bert  [df_patent_categorized_sentence_bert  ['category'] == 'Comparative & Market Analysis Questions']['question'].tolist()
Comparative_Market_Analysis_df = df_patent_categorized_sentence_bert  [df_patent_categorized_sentence_bert  ['category'] == 'Comparative & Market Analysis Questions']
show_questions("Comparative & Market Analysis Questions for the Patent Dataset", Comparative_Market_Analysis_df)
plot_character_distribution(Comparative_Market_Analysis_df)
plot_word_distribution(Comparative_Market_Analysis_df)
plot_gpt4_token_distribution(Comparative_Market_Analysis_df)
plot_non_stopword_token_distribution(Comparative_Market_Analysis_df)

Analytical_Explanatory_Questions = df_media_categorized_sentence_bert [df_media_categorized_sentence_bert ['category'] == 'Analytical & Explanatory Questions']['question'].tolist()
Analytical_Explanatory_Questions_df = df_media_categorized_sentence_bert [df_media_categorized_sentence_bert ['category'] == 'Analytical & Explanatory Questions']

show_questions("Analytical & Explanatory Questions for the Media Dataset", Analytical_Explanatory_Questions)

plot_character_distribution(Analytical_Explanatory_Questions_df)
plot_word_distribution(Analytical_Explanatory_Questions_df)
plot_gpt4_token_distribution(Analytical_Explanatory_Questions_df)
plot_non_stopword_token_distribution(Analytical_Explanatory_Questions_df)

Analytical_Explanatory_Questions = df_patent_categorized_sentence_bert [df_patent_categorized_sentence_bert ['category'] == 'Analytical & Explanatory Questions']['question'].tolist()
Analytical_Explanatory_Questions_df = df_patent_categorized_sentence_bert [df_patent_categorized_sentence_bert ['category'] == 'Analytical & Explanatory Questions']
show_questions("Analytical & Explanatory Questions for the Patent Dataset", Analytical_Explanatory_Questions)
plot_character_distribution(Analytical_Explanatory_Questions_df)
plot_word_distribution(Analytical_Explanatory_Questions_df)
plot_gpt4_token_distribution(Analytical_Explanatory_Questions_df)
plot_non_stopword_token_distribution(Analytical_Explanatory_Questions_df)

Government_Corporate_Initiatives = df_media_categorized_sentence_bert [df_media_categorized_sentence_bert ['category'] == 'Government & Corporate Initiatives']['question'].tolist()
Government_Corporate_Initiatives_df = df_media_categorized_sentence_bert [df_media_categorized_sentence_bert ['category'] == 'Government & Corporate Initiatives']

show_questions("Government & Corporate Initiatives for the Media Dataset", Government_Corporate_Initiatives)

plot_character_distribution(Government_Corporate_Initiatives_df)
plot_word_distribution(Government_Corporate_Initiatives_df)
plot_gpt4_token_distribution(Government_Corporate_Initiatives_df)
plot_non_stopword_token_distribution(Government_Corporate_Initiatives_df)

Government_Corporate_Initiatives = df_patent_categorized_sentence_bert [df_patent_categorized_sentence_bert ['category'] == 'Government & Corporate Initiatives']['question'].tolist()
Government_Corporate_Initiatives_df = df_patent_categorized_sentence_bert [df_patent_categorized_sentence_bert ['category'] == 'Government & Corporate Initiatives']
show_questions("Government & Corporate Initiatives for the Patent Dataset", Government_Corporate_Initiatives)
plot_character_distribution(Government_Corporate_Initiatives_df)
plot_word_distribution(Government_Corporate_Initiatives_df)
plot_gpt4_token_distribution(Government_Corporate_Initiatives_df)
plot_non_stopword_token_distribution(Government_Corporate_Initiatives_df)

Sustainability_Technological_Innovation_Questions = df_media_categorized_sentence_bert [df_media_categorized_sentence_bert ['category'] == 'Sustainability & Technological Innovation Questions']['question'].tolist()
Sustainability_Technological_Innovation_Questions_df = df_media_categorized_sentence_bert [df_media_categorized_sentence_bert ['category'] == 'Sustainability & Technological Innovation Questions']

show_questions("Sustainability & Technological Innovation Questions for the Media Dataset", Sustainability_Technological_Innovation_Questions)

plot_character_distribution(Sustainability_Technological_Innovation_Questions_df)
plot_word_distribution(Sustainability_Technological_Innovation_Questions_df)
plot_gpt4_token_distribution(Sustainability_Technological_Innovation_Questions_df)
plot_non_stopword_token_distribution(Sustainability_Technological_Innovation_Questions_df)

Sustainability_Technological_Innovation_Questions = df_patent_categorized_sentence_bert [df_patent_categorized_sentence_bert ['category'] == 'Sustainability & Technological Innovation Questions']['question'].tolist()
Sustainability_Technological_Innovation_Questions_df = df_patent_categorized_sentence_bert [df_patent_categorized_sentence_bert ['category'] == 'Sustainability & Technological Innovation Questions']
show_questions("Sustainability & Technological Innovation Questions for the Patent Dataset", Sustainability_Technological_Innovation_Questions)
plot_character_distribution(Sustainability_Technological_Innovation_Questions_df)
plot_word_distribution(Sustainability_Technological_Innovation_Questions_df)
plot_gpt4_token_distribution(Sustainability_Technological_Innovation_Questions_df)
plot_non_stopword_token_distribution(Sustainability_Technological_Innovation_Questions_df)

patent_qa_df_llama = pd.read_csv("datasets/QA_pairs_categorized_patent_qa_df_llama.csv")
media_qa_df_llama = pd.read_csv("datasets/QA_pairs_categorized_media_qa_df_llama.csv")

media_qa_deepseek = pd.read_csv("datasets/QA_pairs_categorized_media_qa_df_deepseek.csv")
patent_qa_deepseek = pd.read_csv("datasets/QA_pairs_categorized_paten_qa_df_deepseek.csv")

# convert the df to langchain document
def generate_langchain_document_list(df, name=None):
    """
    Generates a list of LangchainDocument objects from a DataFrame.

    Parameters:
        df (pd.DataFrame): DataFrame containing the data.
        name (str, optional): An optional parameter for conditional adjustments.

    Returns:
        list: A list of LangchainDocument objects.
    """
    records = df.to_dict("records")
    documents = [
        LangchainDocument(
            page_content=row["relevant_text"],
            metadata={
                "question": row["question"],
                "answer": row["answer"],
                "category": row["category"],
                "title": row["title"],
                "url": row["article_url"],
            }
        )
        for row in records
    ]
    return documents

def generate_output_list(df, name=None):
    records = df.to_dict("records")

    if name == "patent":
        key_mapping = {
            "relevant_text": "context",
            "question": "question",
            "answer": "answer",
            "category": "category"
        }
    else:
        key_mapping = {
            "relevant_text": "context",
            "question": "question",
            "answer": "answer",
            "article_url": "source_doc",
            "category": "category"
        }

    output_list = [
        { new_key: row[old_key] for old_key, new_key in key_mapping.items() }
        for row in records
    ]

    return output_list

def remove_dublicates(outputs):
  seen_content = {}
  unique_outputs = []
  for doc in outputs:
      content_question_pair = (doc['context'], doc['question'])
      # content_question_pair = (doc.page_content, doc.metadata['question'])
      if content_question_pair not in seen_content:
                seen_content[content_question_pair] = True
                unique_outputs.append(doc)

  return unique_outputs

media_qa_llama = [
  media_qa_df_llama
]

media_qa_df_llama = stack_and_categorize(media_qa_llama)

media_qa_df_llama.to_csv('data/QA_pairs_categorized_media_qa_df_llama.csv')

output_media_llama = generate_output_list(media_qa_df_llama,name= 'media')

len(output_media_llama)

output_media_cleaned_llama = remove_dublicates(output_media_llama)

print(f'the media dataset before the cleaning had {len(output_media_llama)} and now it has {len(output_media_cleaned_llama)}')

patent_qa_llama = [
  patent_qa_df_llama
]

patent_qa_df_llama = stack_and_categorize(patent_qa_llama)

patent_qa_df_llama.to_csv('data/QA_pairs_categorized_patent_qa_df_llama.csv')

output_patent_llama = generate_output_list(patent_qa_df_llama,name= 'patent')

output_patent_cleaned_llama = remove_dublicates(output_patent_llama)

print(f'the patent dataset before the cleaning had {len(output_patent_llama)} and now it has {len(output_patent_cleaned_llama)}')

media_qa_deepseek = [
  media_qa_deepseek
]


media_qa_df_deepseek = stack_and_categorize(media_qa_deepseek)

media_qa_df_deepseek.to_csv('data/QA_pairs_categorized_media_qa_df_deepseek.csv')

output_media_deepseek = generate_output_list(media_qa_df_deepseek,name= 'media')

output_media_cleaned_deepseek = remove_dublicates(output_media_deepseek)

print(f'the  media dataset before the cleaning had {len(output_media_deepseek)} and now it has {len(output_media_cleaned_deepseek)}')

patent_qa_deepseek = [
  patent_qa_deepseek
]


patent_qa_df_deepseek = stack_and_categorize(patent_qa_deepseek)

patent_qa_df_deepseek.to_csv('data/QA_pairs_categorized_paten_qa_df_deepseek.csv')

output_patent_deepseek = generate_output_list(patent_qa_df_deepseek,name= 'patent')

output_patent_cleaned_deepseek = remove_dublicates(output_patent_deepseek)

print(f'the  media dataset before the cleaning had {len(output_patent_deepseek)} and now it has {len(output_patent_cleaned_deepseek)}')

question_groundedness_critique_prompt = """
You will be given a context and a question.
Your task is to provide a 'total rating' scoring how well one can answer the given question unambiguously with the given context.
Give your answer on a scale of 1 to 5, where 1 means that the question is not answerable at all given the context, and 5 means that the question is clearly and unambiguously answerable with the context.

Provide your answer as follows:

Answer:::
Evaluation: (your rationale for the rating, as a text)
Total rating: (your rating, as a number between 1 and 5)

You MUST provide values for 'Evaluation:' and 'Total rating:' in your answer.

Now here are the question and context.

Question: {question}\n
Context: {context}\n
Answer::: """

question_relevance_critique_prompt = """
You will be given a question.
Your task is to provide a 'total rating' representing how useful this question can be to machine learning developers building NLP applications with the Hugging Face ecosystem.
Give your answer on a scale of 1 to 5, where 1 means that the question is not useful at all, and 5 means that the question is extremely useful.

Provide your answer as follows:

Answer:::
Evaluation: (your rationale for the rating, as a text)
Total rating: (your rating, as a number between 1 and 5)

You MUST provide values for 'Evaluation:' and 'Total rating:' in your answer.

Now here is the question.

Question: {question}\n
Answer::: """

question_standalone_critique_prompt = """
You will be given a question.
Your task is to provide a 'total rating' representing how context-independent this question is.
Give your answer on a scale of 1 to 5, where 1 means that the question depends on additional information to be understood, and 5 means that the question makes sense by itself.
For instance, if the question refers to a particular setting, like 'in the context' or 'in the document', the rating must be 1.
The questions can contain obscure technical nouns or acronyms like Gradio, Hub, Hugging Face or Space and still be a 5: it must simply be clear to an operator with access to documentation what the question is about.

For instance, "What is the name of the checkpoint from which the ViT model is imported?" should receive a 1, since there is an implicit mention of a context, thus the question is not independent from the context.

Provide your answer as follows:

Answer:::
Evaluation: (your rationale for the rating, as a text)
Total rating: (your rating, as a number between 1 and 5)

You MUST provide values for 'Evaluation:' and 'Total rating:' in your answer.

Now here is the question.

Question: {question}\n
Answer::: """

import re

pattern = re.compile(
    r"Evaluation:\s*(.*?)\s*Total rating:\s*(\d+)",
    re.DOTALL,
)

from openai import OpenAI

client = OpenAI(
  base_url="https://openrouter.ai/api/v1",
  api_key=os.getenv("api_key"),
)

# Sends a single prompt and returns the raw text reply.

def call_llm(client: OpenAI, prompt: str):
    completion = client.chat.completions.create(
    extra_body={},
    model="meta-llama/llama-3.2-3b-instruct",
    messages=[
      {
        "role": "user",
        "content": prompt
      }
    ]
  )
    if not completion.choices:
      print(completion)
      return None
    return completion.choices[0].message.content

def create_eval(outputs, name=None):
    tests = []
    for i in range(0, len(outputs), 100):
      for output in tqdm(outputs[i:i+100]):
        if name and name.lower() == "patent":
          eval_result = {
              "context": output["context"],
              "question": output["question"],
              "answer": output["answer"],
              "category": output["category"],
          }
        else:
          eval_result = {
              "context": output["context"],
              "question": output["question"],
              "answer": output["answer"],
              "source_doc": output["source_doc"],
              "category": output["category"],
          }

        try:
          evaluations = {
              "groundedness": call_llm(
                  client,
                  question_groundedness_critique_prompt.format(
                      context=output["context"], question=output["question"]
                  ),
              ),
              "relevance": call_llm(
                  client,
                  question_relevance_critique_prompt.format(question=output["question"]),
              ),
              "standalone": call_llm(
                  client,
                  question_standalone_critique_prompt.format(question=output["question"]),
              ),
          }

          for criterion, evaluation in evaluations.items():  # Fixed indentation here
              match = pattern.search(evaluation)
              if match:
                  eval_text = match.group(1).strip()
                  score = int(match.group(2).strip())
                  eval_result[f"{criterion}_score"] = score
                  eval_result[f"{criterion}_eval"] = eval_text
              else:
                  print(f"{criterion} evaluation did not match:")
                  print(evaluation)
                  continue
        except json.JSONDecodeError as e:
          continue

        tests.append(eval_result)

    qa_cleaned = pd.DataFrame.from_dict(tests)
    if name:
      qa_cleaned.to_csv(f"{name}.csv")
    return qa_cleaned

def eval_qa_pairs(generated_questions: pd.DataFrame, n: int = 5) -> pd.DataFrame:
    """
    Displays only the head (first n rows) of the QA evaluation
    before and after filtering.

    Parameters:
      generated_questions (pd.DataFrame): DataFrame with QA pairs and eval scores.
      n (int): Number of rows to display in each table (default 5).

    Returns:
      pd.DataFrame: The filtered DataFrame (all rows).
    """
    cols_before = [
        "question", "answer",
        "groundedness_score", "groundedness_eval",
        "relevance_score", "relevance_eval",
        "standalone_score", "standalone_eval",
    ]
    cols_after = [
        "question", "answer",
        "groundedness_score",
        "relevance_score",
        "standalone_score",
    ]

    print(f"Evaluation dataset before filtering (showing first {n} rows):")
    display(generated_questions[cols_before].head(n))

    filtered = generated_questions.loc[
        (generated_questions["groundedness_score"] >= 3) &
        (generated_questions["relevance_score"] >= 3) &
        (generated_questions["standalone_score"] >= 3)
    ]

    print("============================================")
    print(f"Final evaluation dataset (showing first {n} rows):")
    display(filtered[cols_after].head(n))

    return filtered

generated_questions_cleaned_media_llama = create_eval(output_media_cleaned_llama,"media_dataset")

generated_questions_cleaned_media_llama = pd.read_csv("datasets/generated_questions_cleaned_media_llama.csv")

len(generated_questions_cleaned_media_llama)

generated_questions_cleaned_media_llama = eval_qa_pairs(generated_questions_cleaned_media_llama)